package tech.abing.spider.task.crawler.amac;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;

import tech.abing.spider.CrawTaskBuilder;
import tech.abing.spider.CrawlManager;
import tech.abing.spider.CrawlTask;
import tech.abing.spider.component.DynamicEntrance;
import tech.abing.spider.component.listener.TaskLifeListener;
import tech.abing.spider.downloader.impl.DefaultPageDownloader;
import tech.abing.spider.request.PageRequest;
import tech.abing.spider.request.PageRequest.PageEncoding;
import tech.abing.spider.request.StartContext;
import tech.abing.spider.task.constants.DictEnum;
import tech.abing.spider.task.pageprocessor.amac.AdminLicenseProcessor;
import tech.abing.spider.task.pageprocessor.amac.BlackListProcessor;
import tech.abing.spider.task.pipeline.amac.AdminLicensePipeline;
import tech.abing.spider.util.SqlUtil;

public class AdminLicenseSpider {
	
	private static SqlUtil sqlUtil = null;
	
	public static void start() {
		CrawlManager manager = CrawlManager.getInstance();
		
		CrawTaskBuilder builder = manager
		.prepareCrawlTask(DictEnum.TASK_TYPE_ADMIN_LICENSE.getName(), DefaultPageDownloader.class)
		.useThread(10)
		.useDynamicEntrance(DynamicEntranceImpl.class)
		.usePageEncoding(PageEncoding.UTF8)
		.usePageRetryCount(3)
		.setPolitenessDelay(1000)
		.usePipeline(AdminLicensePipeline.class)
		.useTaskLifeListener(new TaskLifeListener(){

			@Override
			public void onStart(CrawlTask task) {
				System.out.println("================开始");
				if(sqlUtil==null){
					sqlUtil = SqlUtil.getInstance();
				}
				sqlUtil.deleteForConn("delete from "+DictEnum.TASK_TYPE_ADMIN_LICENSE.getCode());
				sqlUtil.deleteForConn("delete from "+DictEnum.TASK_TYPE_ADMIN_LICENSE_DETAIL.getCode());
//				sqlUtil.deleteForConn("delete from "+DictEnum.TASK_TYPE_ATTACHMENT.getCode()+" where 1=1 and ATTACH_TYPE = ?",TaskConstants.TASK_BLACKLIST);
			}

			@Override
			public void onFinished(CrawlTask task) {
				task.getPipeline().destory();
				sqlUtil.closeConn();
				sqlUtil = null;
				System.out.println("================完成");
			}
			
		});
		
		CrawlTask task = builder.build();
		
		manager.start(task);
	}
	
//	public static final class DynamicEntranceImpl extends DynamicEntrance{
//
//		@Override
//		public List<StartContext> loadStartContext() {
//			StartContext context = new StartContext();
//			String url = "http://www.csrc.gov.cn/pub/zjhpublicofbj/3284/3566/index_887.htm";
//			
//			PageRequest req = context.createPageRequest(url, AdminLicenseProcessor.class);
//			req.putHeader("User-Agent", "Mozilla/5.0 (iPhone; CPU iPhone OS 9_1 like Mac OS X) AppleWebKit/601.1.46 (KHTML, like Gecko) Version/9.0 Mobile/13B143 Safari/601.1");
//			req.putHeader("Referer", "http://www.csrc.gov.cn/pub/zjhpublicofbj/3284/3566/index_887_1.htm");
//			req.putHeader("Host", "www.csrc.gov.cn");
//			req.putHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
//			req.putHeader("Accept-Language", "zh-CN,zh;q=0.8");
//			req.putHeader("Cache-Control", "max-age=0");
//			req.putHeader("Upgrade-Insecure-Requests", "1");
//			req.putHeader("Cookie", "_gscu_516223281=0164042185ybd321; _gscs_516223281=t01825789wkke8k24|pv:1; _gscbrs_516223281=1");
////			req.putHeader("If-Modified-Since", "Fri, 28 Jul 2017 10:07:08 GMT");
//			req.putHeader("If-None-Match", "630000000455c7-eaed-5555dd80d1091");
//			
//			context.injectSeed(req);
//			
//			return Arrays.asList(context);
//		}
//		
//	}

	public static final class DynamicEntranceImpl extends DynamicEntrance{

		@Override
		public List<StartContext> loadStartContext() {
			StartContext context = new StartContext();
			List<String> urlList = new ArrayList<String>();
			urlList.add("http://www.csrc.gov.cn/pub/zjhpublicofbj/3284/3566/index_887.htm");
			urlList.add("http://www.csrc.gov.cn/pub/zjhpublicoftj/3284/3566/index_1008.htm");
			urlList.add("http://www.csrc.gov.cn/pub/zjhpublicofheb/3284/3566/index_1030.htm");
			urlList.add("http://www.csrc.gov.cn/pub/zjhpublicofsx/3284/3566/index_1041.htm");
			urlList.add("http://www.csrc.gov.cn/pub/zjhpublicofnmg/3284/3566/index_1052.htm");
			urlList.add("http://www.csrc.gov.cn/pub/zjhpublicofln/3284/3566/index_1063.htm");
			urlList.add("http://www.csrc.gov.cn/pub/zjhpublicofjl/3284/3566/index_1074.htm");
			urlList.add("http://www.csrc.gov.cn/pub/zjhpublicofhlj/3284/3566/index_1085.htm");
			urlList.add("http://www.csrc.gov.cn/pub/zjhpublicofsh/3284/3566/index_1239.htm");
			
			urlList.add("http://www.csrc.gov.cn/pub/zjhpublicofjs/3284/3566/index_1228.htm");
			urlList.add("http://www.csrc.gov.cn/pub/zjhpublicofzj/3284/3566/index_1096.htm");
			urlList.add("http://www.csrc.gov.cn/pub/zjhpublicofah/3284/3566/index_1019.htm");
			
			urlList.add("http://www.csrc.gov.cn/pub/zjhpublicoffj/3284/3566/index_1129.htm");
			urlList.add("http://www.csrc.gov.cn/pub/zjhpublicofsd/3284/3566/index_997.htm");
			urlList.add("http://www.csrc.gov.cn/pub/zjhpublicofjx/3284/3566/index_964.htm");
			
			urlList.add("http://www.csrc.gov.cn/pub/zjhpublicofhb/3284/3566/index_942.htm");
			urlList.add("http://www.csrc.gov.cn/pub/zjhpublicofhn/3284/3566/index_953.htm");
			urlList.add("http://www.csrc.gov.cn/pub/zjhpublicofhen/3284/3566/index_986.htm");
			
			urlList.add("http://www.csrc.gov.cn/pub/zjhpublicofgd/3284/3566/index_832.htm");			
			urlList.add("http://www.csrc.gov.cn/pub/zjhpublicofgx/3284/3566/index_1140.htm");
			urlList.add("http://www.csrc.gov.cn/pub/zjhpublicofhan/3284/3566/index_1162.htm");
			
			urlList.add("http://www.csrc.gov.cn/pub/zjhpublicofcq/3284/3566/index_898.htm");
			urlList.add("http://www.csrc.gov.cn/pub/zjhpublicofsc/3284/3566/index_1173.htm");
			urlList.add("http://www.csrc.gov.cn/pub/zjhpublicofgz/3284/3566/index_1184.htm");
			
			urlList.add("http://www.csrc.gov.cn/pub/zjhpublicofyn/3284/3566/index_1195.htm");
			urlList.add("http://www.csrc.gov.cn/pub/zjhpublicofxz/3284/3566/index_1206.htm");
			urlList.add("http://www.csrc.gov.cn/pub/zjhpublicofsax/3284/3566/index_1217.htm");
			
			urlList.add("http://www.csrc.gov.cn/pub/zjhpublicofgs/3284/3566/index_1118.htm");
			urlList.add("http://www.csrc.gov.cn/pub/zjhpublicofqh/3284/3566/index_931.htm");
			urlList.add("http://www.csrc.gov.cn/pub/zjhpublicofnx/3284/3566/index_920.htm");
			
			urlList.add("http://www.csrc.gov.cn/pub/zjhpublicofxj/3284/3566/index_909.htm");
			urlList.add("http://www.csrc.gov.cn/pub/zjhpublicofsz/3284/3566/index_876.htm");
			urlList.add("http://www.csrc.gov.cn/pub/zjhpublicofdl/3284/3566/index_1250.htm");
			
			urlList.add("http://www.csrc.gov.cn/pub/zjhpublicofnb/3284/3566/index_1107.htm");
			urlList.add("http://www.csrc.gov.cn/pub/zjhpublicofxm/3284/3566/index_1151.htm");
			urlList.add("http://www.csrc.gov.cn/pub/zjhpublicofqd/3284/3566/index_975.htm");
			// 通知公告
			urlList.add("http://www.csrc.gov.cn/pub/zjhpublicofbj/3284/3565/index_887.htm");
			urlList.add("http://www.csrc.gov.cn/pub/zjhpublicoftj/3284/3565/index_1008.htm");
			urlList.add("http://www.csrc.gov.cn/pub/zjhpublicofheb/3284/3565/index_1030.htm");
			urlList.add("http://www.csrc.gov.cn/pub/zjhpublicofsx/3284/3565/index_1041.htm");
			urlList.add("http://www.csrc.gov.cn/pub/zjhpublicofnmg/3284/3565/index_1052.htm");
			urlList.add("http://www.csrc.gov.cn/pub/zjhpublicofln/3284/3565/index_1063.htm");
			urlList.add("http://www.csrc.gov.cn/pub/zjhpublicofjl/3284/3565/index_1074.htm");
			urlList.add("http://www.csrc.gov.cn/pub/zjhpublicofhlj/3284/3565/index_1085.htm");
			urlList.add("http://www.csrc.gov.cn/pub/zjhpublicofsh/3284/3565/index_1239.htm");
			
			urlList.add("http://www.csrc.gov.cn/pub/zjhpublicofjs/3284/3565/index_1228.htm");
			urlList.add("http://www.csrc.gov.cn/pub/zjhpublicofzj/3284/3565/index_1096.htm");
			urlList.add("http://www.csrc.gov.cn/pub/zjhpublicofah/3284/3565/index_1019.htm");
			
			urlList.add("http://www.csrc.gov.cn/pub/zjhpublicoffj/3284/3565/index_1129.htm");
			urlList.add("http://www.csrc.gov.cn/pub/zjhpublicofsd/3284/3565/index_997.htm");
			urlList.add("http://www.csrc.gov.cn/pub/zjhpublicofjx/3284/3565/index_964.htm");
			
			urlList.add("http://www.csrc.gov.cn/pub/zjhpublicofhb/3284/3565/index_942.htm");
			urlList.add("http://www.csrc.gov.cn/pub/zjhpublicofhn/3284/3565/index_953.htm");
			urlList.add("http://www.csrc.gov.cn/pub/zjhpublicofhen/3284/3565/index_986.htm");
			
			urlList.add("http://www.csrc.gov.cn/pub/zjhpublicofgd/3284/3565/index_832.htm");			
			urlList.add("http://www.csrc.gov.cn/pub/zjhpublicofgx/3284/3565/index_1140.htm");
			urlList.add("http://www.csrc.gov.cn/pub/zjhpublicofhan/3284/3565/index_1162.htm");
			
			urlList.add("http://www.csrc.gov.cn/pub/zjhpublicofcq/3284/3565/index_898.htm");
			urlList.add("http://www.csrc.gov.cn/pub/zjhpublicofsc/3284/3565/index_1173.htm");
			urlList.add("http://www.csrc.gov.cn/pub/zjhpublicofgz/3284/3565/index_1184.htm");
			
			urlList.add("http://www.csrc.gov.cn/pub/zjhpublicofyn/3284/3565/index_1195.htm");
			urlList.add("http://www.csrc.gov.cn/pub/zjhpublicofxz/3284/3565/index_1206.htm");
			urlList.add("http://www.csrc.gov.cn/pub/zjhpublicofsax/3284/3565/index_1217.htm");
			
			urlList.add("http://www.csrc.gov.cn/pub/zjhpublicofgs/3284/3565/index_1118.htm");
			urlList.add("http://www.csrc.gov.cn/pub/zjhpublicofqh/3284/3565/index_931.htm");
			urlList.add("http://www.csrc.gov.cn/pub/zjhpublicofnx/3284/3565/index_920.htm");
			
			urlList.add("http://www.csrc.gov.cn/pub/zjhpublicofxj/3284/3565/index_909.htm");
			urlList.add("http://www.csrc.gov.cn/pub/zjhpublicofsz/3284/3565/index_876.htm");
			urlList.add("http://www.csrc.gov.cn/pub/zjhpublicofdl/3284/3565/index_1250.htm");
			
			urlList.add("http://www.csrc.gov.cn/pub/zjhpublicofnb/3284/3565/index_1107.htm");
			urlList.add("http://www.csrc.gov.cn/pub/zjhpublicofxm/3284/3565/index_1151.htm");
			urlList.add("http://www.csrc.gov.cn/pub/zjhpublicofqd/3284/3565/index_975.htm");
			
			
			for (String url : urlList) {
				PageRequest req = buildReq(context, url);
				context.injectSeed(req);
			}
			
			return Arrays.asList(context);
		}
		
		private PageRequest buildReq(StartContext context, String url) {
			PageRequest req = context.createPageRequest(url, AdminLicenseProcessor.class);
			req.putHeader("User-Agent", "Mozilla/5.0 (iPhone; CPU iPhone OS 9_1 like Mac OS X) AppleWebKit/601.1.46 (KHTML, like Gecko) Version/9.0 Mobile/13B143 Safari/601.1");
			req.putHeader("Referer", "http://www.csrc.gov.cn/pub/zjhpublicofbj/3284/3566/index_887_1.htm");
			req.putHeader("Host", "www.csrc.gov.cn");
			req.putHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
			req.putHeader("Accept-Language", "zh-CN,zh;q=0.8");
			req.putHeader("Cache-Control", "max-age=0");
			req.putHeader("Upgrade-Insecure-Requests", "1");
			req.putHeader("Cookie", "_gscu_516223281=0164042185ybd321; _gscs_516223281=t01825789wkke8k24|pv:1; _gscbrs_516223281=1");
//			req.putHeader("If-Modified-Since", "Fri, 28 Jul 2017 10:07:08 GMT");
			req.putHeader("If-None-Match", "630000000455c7-eaed-5555dd80d1091");
			
			return req;
		}
		
	}
}
